In [1]:
from tensorflow.keras import backend as K
import gc
K.clear_session()
gc.collect()

WARNING:tensorflow:From C:\Users\Nidhi\anaconda3\envs\py311\Lib\site-packages\keras\src\backend\common\global_state.py:82: The name tf.reset_default_graph is deprecated. Please use tf.compat.v1.reset_default_graph instead.
Out[1]:
0

Multiclass classification¶

Step1: Data Preparation¶

In [1]:
import numpy as np
import matplotlib.pyplot as plt
import h5py
import tensorflow as tf
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
from tensorflow.keras.utils import to_categorical
from sklearn.utils import class_weight
from tensorflow.keras.metrics import Precision, Recall, AUC
from keras.models import Sequential
from keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout
In [2]:
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
In [3]:
# Load the HDF5 file
bird_sound = h5py.File("C:\\Users\\Nidhi\\Documents\\College Material\\Spring Quarter 2025\\SML2\\Homework3\\bird_spectrograms.hdf5", 'r')
In [4]:
species= list(bird_sound.keys())

# Load all spectrograms and labels
X = []
y = []

for label in species:
    data =bird_sound[label][:]  
    data = np.transpose(data, (2, 0, 1))  
    X.append(data)
    y += [label] * data.shape[0]


X = np.concatenate(X, axis=0)
print(X.shape) 

(1981, 128, 517)
In [5]:
# Plot one spectrogram sample from each species
unique_species = list(np.unique(y))
sample_specs = {}

for label in unique_species:
    label_index = y.index(label) if isinstance(y, list) else np.where(np.array(y) == label)[0][0]
    sample = X[label_index, :, :]

    sample_specs[label] = sample

plt.figure(figsize=(15, 3 * len(sample_specs)))

for i, (label, spec) in enumerate(sample_specs.items()):
    plt.subplot(len(sample_specs), 1, i + 1)
    plt.imshow(spec, aspect='auto', origin='lower', cmap='inferno')
    plt.title(f"Species: {label}")
    plt.colorbar()
    plt.axis('off')

plt.tight_layout()
plt.show()
No description has been provided for this image

Step2: Train-Test Split¶

In [6]:
X = X.reshape((1981, 128, 517, 1))
y = np.array(y)
label_encoder = LabelEncoder()
y_encoded = label_encoder.fit_transform(y)
y_categorical = to_categorical(y_encoded)


# Train-test split
X_train, X_test, y_train, y_test = train_test_split(
    X, y_categorical, test_size=0.2, random_state=156, stratify=y_encoded
)
print(f"X_train shape: {X_train.shape}, y_train shape: {y_train.shape}")
print(f"X_test shape: {X_test.shape}, y_test shape: {y_test.shape}")
print(f"Number of classes: {len(label_encoder.classes_)}")

X_train shape: (1584, 128, 517, 1), y_train shape: (1584, 12)
X_test shape: (397, 128, 517, 1), y_test shape: (397, 12)
Number of classes: 12

Step3: Model Training¶

Model1¶
In [8]:
# Definigng model1
np.random.seed(123)
tf.random.set_seed(123)

model1_multi = Sequential()
model1_multi.add(Conv2D(32, (3, 3), activation='relu', input_shape=(128, 517, 1)))
model1_multi.add(MaxPooling2D(pool_size=(2, 2)))

model1_multi.add(Conv2D(64, (3, 3), activation='relu'))
model1_multi.add(MaxPooling2D(pool_size=(2, 2)))

model1_multi.add(Flatten())
model1_multi.add(Dense(128, activation='relu'))
model1_multi.add(Dropout(0.5))

model1_multi.add(Dense(12, activation='softmax'))  


model1_multi.compile(
    optimizer='adam',
    loss='categorical_crossentropy',
    metrics=[
        'accuracy',
        Precision(name='precision'),
        Recall(name='recall'),
        AUC(name='auc')
    ]
)

C:\Users\Nidhi\anaconda3\envs\py311\Lib\site-packages\keras\src\layers\convolutional\base_conv.py:107: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)
In [9]:
import time
start_time = time.time()

history1 = model1_multi.fit(
    X_train, y_train,
    validation_data=(X_test, y_test),
    epochs=20,
    batch_size=256
)

end_time = time.time()
time1 = end_time - start_time
print(f"\nTraining time: {time1:.2f} seconds")

Epoch 1/20
7/7 ━━━━━━━━━━━━━━━━━━━━ 34s 5s/step - accuracy: 0.1411 - auc: 0.5307 - loss: 545.9827 - precision: 0.1426 - recall: 0.1411 - val_accuracy: 0.0453 - val_auc: 0.4780 - val_loss: 59.9108 - val_precision: 0.0455 - val_recall: 0.0453
Epoch 2/20
7/7 ━━━━━━━━━━━━━━━━━━━━ 31s 4s/step - accuracy: 0.1031 - auc: 0.5173 - loss: 44.2694 - precision: 0.1022 - recall: 0.0866 - val_accuracy: 0.3073 - val_auc: 0.5002 - val_loss: 2.4888 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00
Epoch 3/20
7/7 ━━━━━━━━━━━━━━━━━━━━ 34s 5s/step - accuracy: 0.2602 - auc: 0.5037 - loss: 3.2944 - precision: 0.0763 - recall: 0.0186 - val_accuracy: 0.0982 - val_auc: 0.4269 - val_loss: 2.9123 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00
Epoch 4/20
7/7 ━━━━━━━━━━━━━━━━━━━━ 31s 4s/step - accuracy: 0.1834 - auc: 0.5233 - loss: 2.7006 - precision: 0.0000e+00 - recall: 0.0000e+00 - val_accuracy: 0.3476 - val_auc: 0.6964 - val_loss: 2.4225 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00
Epoch 5/20
7/7 ━━━━━━━━━━━━━━━━━━━━ 32s 5s/step - accuracy: 0.3829 - auc: 0.7179 - loss: 2.3431 - precision: 0.3555 - recall: 0.0182 - val_accuracy: 0.3602 - val_auc: 0.7486 - val_loss: 2.2619 - val_precision: 1.0000 - val_recall: 0.0025
Epoch 6/20
7/7 ━━━━━━━━━━━━━━━━━━━━ 32s 5s/step - accuracy: 0.4012 - auc: 0.7967 - loss: 2.0732 - precision: 0.8099 - recall: 0.0506 - val_accuracy: 0.4030 - val_auc: 0.7713 - val_loss: 2.0368 - val_precision: 0.5932 - val_recall: 0.0882
Epoch 7/20
7/7 ━━━━━━━━━━━━━━━━━━━━ 32s 5s/step - accuracy: 0.4880 - auc: 0.8424 - loss: 1.7971 - precision: 0.6669 - recall: 0.1303 - val_accuracy: 0.4081 - val_auc: 0.7851 - val_loss: 2.0887 - val_precision: 0.5025 - val_recall: 0.2569
Epoch 8/20
7/7 ━━━━━━━━━━━━━━━━━━━━ 32s 4s/step - accuracy: 0.5516 - auc: 0.8821 - loss: 1.5779 - precision: 0.7027 - recall: 0.2674 - val_accuracy: 0.4131 - val_auc: 0.7977 - val_loss: 1.9996 - val_precision: 0.5183 - val_recall: 0.2141
Epoch 9/20
7/7 ━━━━━━━━━━━━━━━━━━━━ 33s 5s/step - accuracy: 0.6463 - auc: 0.9291 - loss: 1.2091 - precision: 0.7768 - recall: 0.4323 - val_accuracy: 0.3703 - val_auc: 0.7754 - val_loss: 2.2326 - val_precision: 0.4770 - val_recall: 0.2091
Epoch 10/20
7/7 ━━━━━━━━━━━━━━━━━━━━ 34s 5s/step - accuracy: 0.7044 - auc: 0.9524 - loss: 1.0085 - precision: 0.8508 - recall: 0.4959 - val_accuracy: 0.4257 - val_auc: 0.8066 - val_loss: 2.0428 - val_precision: 0.5694 - val_recall: 0.2997
Epoch 11/20
7/7 ━━━━━━━━━━━━━━━━━━━━ 33s 5s/step - accuracy: 0.7876 - auc: 0.9773 - loss: 0.7180 - precision: 0.9011 - recall: 0.6416 - val_accuracy: 0.4408 - val_auc: 0.8011 - val_loss: 2.0789 - val_precision: 0.5631 - val_recall: 0.2922
Epoch 12/20
7/7 ━━━━━━━━━━━━━━━━━━━━ 32s 5s/step - accuracy: 0.8431 - auc: 0.9899 - loss: 0.5213 - precision: 0.9326 - recall: 0.7390 - val_accuracy: 0.4332 - val_auc: 0.8012 - val_loss: 2.1565 - val_precision: 0.5402 - val_recall: 0.3048
Epoch 13/20
7/7 ━━━━━━━━━━━━━━━━━━━━ 32s 5s/step - accuracy: 0.9085 - auc: 0.9956 - loss: 0.3503 - precision: 0.9638 - recall: 0.8365 - val_accuracy: 0.4559 - val_auc: 0.7993 - val_loss: 2.3629 - val_precision: 0.5349 - val_recall: 0.3476
Epoch 14/20
7/7 ━━━━━━━━━━━━━━━━━━━━ 31s 4s/step - accuracy: 0.9285 - auc: 0.9976 - loss: 0.2582 - precision: 0.9752 - recall: 0.8818 - val_accuracy: 0.4307 - val_auc: 0.8021 - val_loss: 2.4970 - val_precision: 0.5068 - val_recall: 0.3778
Epoch 15/20
7/7 ━━━━━━━━━━━━━━━━━━━━ 32s 5s/step - accuracy: 0.9570 - auc: 0.9992 - loss: 0.1659 - precision: 0.9755 - recall: 0.9340 - val_accuracy: 0.4408 - val_auc: 0.8051 - val_loss: 2.5254 - val_precision: 0.4765 - val_recall: 0.3577
Epoch 16/20
7/7 ━━━━━━━━━━━━━━━━━━━━ 31s 4s/step - accuracy: 0.9673 - auc: 0.9994 - loss: 0.1530 - precision: 0.9851 - recall: 0.9394 - val_accuracy: 0.4257 - val_auc: 0.7952 - val_loss: 2.8406 - val_precision: 0.4763 - val_recall: 0.3804
Epoch 17/20
7/7 ━━━━━━━━━━━━━━━━━━━━ 31s 4s/step - accuracy: 0.9562 - auc: 0.9993 - loss: 0.1310 - precision: 0.9726 - recall: 0.9422 - val_accuracy: 0.4710 - val_auc: 0.8041 - val_loss: 2.5447 - val_precision: 0.5288 - val_recall: 0.3929
Epoch 18/20
7/7 ━━━━━━━━━━━━━━━━━━━━ 31s 4s/step - accuracy: 0.9813 - auc: 0.9996 - loss: 0.0987 - precision: 0.9884 - recall: 0.9640 - val_accuracy: 0.4332 - val_auc: 0.8000 - val_loss: 2.6519 - val_precision: 0.4901 - val_recall: 0.3753
Epoch 19/20
7/7 ━━━━━━━━━━━━━━━━━━━━ 31s 4s/step - accuracy: 0.9797 - auc: 0.9997 - loss: 0.0756 - precision: 0.9891 - recall: 0.9743 - val_accuracy: 0.4257 - val_auc: 0.7952 - val_loss: 2.9259 - val_precision: 0.4985 - val_recall: 0.4106
Epoch 20/20
7/7 ━━━━━━━━━━━━━━━━━━━━ 31s 4s/step - accuracy: 0.9854 - auc: 0.9999 - loss: 0.0571 - precision: 0.9896 - recall: 0.9803 - val_accuracy: 0.4332 - val_auc: 0.7920 - val_loss: 2.7538 - val_precision: 0.4903 - val_recall: 0.3829

Training time: 641.04 seconds
In [10]:
print("\nValidation Accuracy Model1:")
for m in ['val_accuracy', 'val_precision', 'val_recall', 'val_auc']:
    print(f"{m}: {history1.history[m][-1]:.4f}")

Validation Accuracy Model1:
val_accuracy: 0.4332
val_precision: 0.4903
val_recall: 0.3829
val_auc: 0.7920
  • Model 1 has validation accuracy of (43.32%) and a strong AUC of 0.7844. These results indicate effective overall classification across multiple bird species.
In [11]:
# plotting the results
plt.figure(figsize=(12, 5))

plt.subplot(1, 2, 1)
plt.plot(history1.history['accuracy'], label='Train Accuracy')
plt.plot(history1.history['val_accuracy'], label='Val Accuracy')
plt.title('Model Accuracy')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.legend()

plt.subplot(1, 2, 2)
plt.plot(history1.history['loss'], label='Train Loss')
plt.plot(history1.history['val_loss'], label='Val Loss')
plt.title('Model Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()

plt.tight_layout()
plt.show()
No description has been provided for this image
  • The graphs show that the model's training accuracy keeps improving, but the validation accuracy stays around 45%, which means the model might be overfitting.
  • The loss graph also shows training loss goes down , while validation loss remains mostly flat indicatingthat the model is learning the training data well but not generalizing to new data.
In [12]:
from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay
import matplotlib.pyplot as plt

y_pred = model1_multi.predict(X_test)
y_pred_classes = np.argmax(y_pred, axis=1)
y_true = np.argmax(y_test, axis=1)

cm1 = confusion_matrix(y_true, y_pred_classes)
plt.figure(figsize=(8, 6))
sns.heatmap(cm1, annot=True, fmt='d', cmap='Blues',
            xticklabels=label_encoder.classes_,
            yticklabels=label_encoder.classes_)

plt.xlabel('Predicted')
plt.ylabel('True')
plt.title('Confusion Matrix Heatmap')
plt.xticks(rotation=90)
plt.yticks(rotation=0)
plt.tight_layout()
plt.show()

13/13 ━━━━━━━━━━━━━━━━━━━━ 2s 117ms/step
No description has been provided for this image
  • This confusion matrix shows that the model performs best on houspa (House Sparrow), predicting it correctly 106 times. However, it struggles to distinguish between several other bird species like bewwre, rewbla, and sonspa, often confusing them with each other.
Model2¶
In [13]:
# Defining model2
np.random.seed(42)
tf.random.set_seed(42)


model2_multi = Sequential()
model2_multi.add(Conv2D(16, (3, 3), activation='relu', input_shape=(128, 517, 1)))
model2_multi.add(MaxPooling2D(pool_size=(2, 2)))

model2_multi.add(Conv2D(32, (3, 3), activation='relu'))
model2_multi.add(MaxPooling2D(pool_size=(2, 2)))

model2_multi.add(Flatten())
model2_multi.add(Dense(64, activation='relu'))
model2_multi.add(Dropout(0.3))

model2_multi.add(Dense(12, activation='softmax'))  

model2_multi.compile(
    optimizer='adam',
    loss='categorical_crossentropy',
    metrics=[
        'accuracy',
        Precision(name='precision'),
        Recall(name='recall'),
        AUC(name='auc')
    ]
)

C:\Users\Nidhi\anaconda3\envs\py311\Lib\site-packages\keras\src\layers\convolutional\base_conv.py:107: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)
In [14]:
# Tracking the model training time
import time
start_time = time.time()
history2 = model2_multi.fit(
    X_train, y_train,
    validation_data=(X_test, y_test),
    epochs=15,
    batch_size=128
)

end_time = time.time()
time2 = end_time - start_time
print(f"\nTraining time: {time2:.2f} seconds")

Epoch 1/15
13/13 ━━━━━━━━━━━━━━━━━━━━ 11s 694ms/step - accuracy: 0.1675 - auc: 0.5497 - loss: 225.6911 - precision: 0.1647 - recall: 0.1556 - val_accuracy: 0.3174 - val_auc: 0.6720 - val_loss: 2.3636 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00
Epoch 2/15
13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 654ms/step - accuracy: 0.1788 - auc: 0.6281 - loss: 2.4847 - precision: 0.0977 - recall: 0.0025 - val_accuracy: 0.2771 - val_auc: 0.6823 - val_loss: 2.3905 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00
Epoch 3/15
13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 670ms/step - accuracy: 0.2556 - auc: 0.6828 - loss: 2.4334 - precision: 0.3094 - recall: 0.0339 - val_accuracy: 0.3149 - val_auc: 0.7282 - val_loss: 2.2707 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00
Epoch 4/15
13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 681ms/step - accuracy: 0.3738 - auc: 0.7624 - loss: 2.1663 - precision: 0.5728 - recall: 0.0493 - val_accuracy: 0.3778 - val_auc: 0.7536 - val_loss: 2.1355 - val_precision: 0.3929 - val_recall: 0.0554
Epoch 5/15
13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 676ms/step - accuracy: 0.5174 - auc: 0.8676 - loss: 1.7304 - precision: 0.6833 - recall: 0.1256 - val_accuracy: 0.4081 - val_auc: 0.7968 - val_loss: 1.9734 - val_precision: 0.5104 - val_recall: 0.1234
Epoch 6/15
13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 668ms/step - accuracy: 0.6408 - auc: 0.9278 - loss: 1.2562 - precision: 0.8154 - recall: 0.3680 - val_accuracy: 0.4131 - val_auc: 0.7979 - val_loss: 2.0009 - val_precision: 0.5152 - val_recall: 0.1285
Epoch 7/15
13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 661ms/step - accuracy: 0.7841 - auc: 0.9761 - loss: 0.7922 - precision: 0.9134 - recall: 0.5615 - val_accuracy: 0.4307 - val_auc: 0.7871 - val_loss: 2.3146 - val_precision: 0.4957 - val_recall: 0.2897
Epoch 8/15
13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 653ms/step - accuracy: 0.8689 - auc: 0.9894 - loss: 0.4926 - precision: 0.9371 - recall: 0.7634 - val_accuracy: 0.4358 - val_auc: 0.8013 - val_loss: 2.2728 - val_precision: 0.4791 - val_recall: 0.3174
Epoch 9/15
13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 661ms/step - accuracy: 0.9118 - auc: 0.9940 - loss: 0.3379 - precision: 0.9584 - recall: 0.8525 - val_accuracy: 0.4282 - val_auc: 0.7932 - val_loss: 2.2156 - val_precision: 0.5138 - val_recall: 0.2821
Epoch 10/15
13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 671ms/step - accuracy: 0.9505 - auc: 0.9960 - loss: 0.2284 - precision: 0.9681 - recall: 0.9132 - val_accuracy: 0.4005 - val_auc: 0.7658 - val_loss: 2.6874 - val_precision: 0.4648 - val_recall: 0.2997
Epoch 11/15
13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 693ms/step - accuracy: 0.9315 - auc: 0.9961 - loss: 0.3092 - precision: 0.9578 - recall: 0.8611 - val_accuracy: 0.4207 - val_auc: 0.7729 - val_loss: 2.5517 - val_precision: 0.4835 - val_recall: 0.3325
Epoch 12/15
13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 686ms/step - accuracy: 0.9635 - auc: 0.9990 - loss: 0.1784 - precision: 0.9795 - recall: 0.9316 - val_accuracy: 0.4106 - val_auc: 0.7864 - val_loss: 2.6806 - val_precision: 0.4804 - val_recall: 0.3401
Epoch 13/15
13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 687ms/step - accuracy: 0.9716 - auc: 0.9991 - loss: 0.1314 - precision: 0.9805 - recall: 0.9521 - val_accuracy: 0.3980 - val_auc: 0.7801 - val_loss: 2.9548 - val_precision: 0.4441 - val_recall: 0.3300
Epoch 14/15
13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 671ms/step - accuracy: 0.9817 - auc: 0.9995 - loss: 0.0800 - precision: 0.9859 - recall: 0.9755 - val_accuracy: 0.4257 - val_auc: 0.7916 - val_loss: 2.8339 - val_precision: 0.5017 - val_recall: 0.3728
Epoch 15/15
13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 668ms/step - accuracy: 0.9873 - auc: 0.9997 - loss: 0.0545 - precision: 0.9915 - recall: 0.9847 - val_accuracy: 0.4635 - val_auc: 0.7810 - val_loss: 3.0003 - val_precision: 0.5081 - val_recall: 0.3929

Training time: 134.07 seconds
In [15]:
print("\nValidation Accuracy Model2:")
for m in ['val_accuracy', 'val_precision', 'val_recall', 'val_auc']:
    print(f"{m}: {history2.history[m][-1]:.4f}")

Validation Accuracy Model2:
val_accuracy: 0.4635
val_precision: 0.5081
val_recall: 0.3929
val_auc: 0.7810
  • Model 2 has validation accuracy of (46.35%) and a strong AUC of 0.7810. These results shows that it has 3% moere accuracy than model1.
In [16]:
plt.figure(figsize=(12, 5))

plt.subplot(1, 2, 1)
plt.plot(history2.history['accuracy'], label='Train Accuracy')
plt.plot(history2.history['val_accuracy'], label='Val Accuracy')
plt.title('Model Accuracy')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.legend()

plt.subplot(1, 2, 2)
plt.plot(history2.history['loss'], label='Train Loss')
plt.plot(history2.history['val_loss'], label='Val Loss')
plt.title('Model Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()

plt.tight_layout()
plt.show()
No description has been provided for this image
In [17]:
from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay
import matplotlib.pyplot as plt

y_pred = model2_multi.predict(X_test)
y_pred_classes = np.argmax(y_pred, axis=1)
y_true = np.argmax(y_test, axis=1)

cm2 = confusion_matrix(y_true, y_pred_classes)
plt.figure(figsize=(8, 6))
sns.heatmap(cm2, annot=True, fmt='d', cmap='Blues',
            xticklabels=label_encoder.classes_,
            yticklabels=label_encoder.classes_)

plt.xlabel('Predicted')
plt.ylabel('True')
plt.title('Confusion Matrix Heatmap')
plt.xticks(rotation=90)
plt.yticks(rotation=0)
plt.tight_layout()
plt.show()

13/13 ━━━━━━━━━━━━━━━━━━━━ 1s 48ms/step
No description has been provided for this image
  • The model predicts House Sparrow and song sparrow very accurately, with minimal confusion.

Model1-A¶

In [18]:
np.random.seed(123)
tf.random.set_seed(123)

model1a_multi = Sequential()
model1a_multi.add(Conv2D(32, (3, 3), activation='relu', input_shape=(128, 517, 1)))
model1a_multi.add(MaxPooling2D(pool_size=(2, 2)))

model1a_multi.add(Conv2D(64, (3, 3), activation='relu'))
model1a_multi.add(MaxPooling2D(pool_size=(2, 2)))

model1a_multi.add(Flatten())
model1a_multi.add(Dense(128, activation='relu'))
model1a_multi.add(Dropout(0.4))

model1a_multi.add(Dense(12, activation='softmax'))  

model1a_multi.compile(
    optimizer='adam',
    loss='categorical_crossentropy',
    metrics=[
        'accuracy',
        Precision(name='precision'),
        Recall(name='recall'),
        AUC(name='auc')
    ]
)

C:\Users\Nidhi\anaconda3\envs\py311\Lib\site-packages\keras\src\layers\convolutional\base_conv.py:107: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)
In [19]:
start_time = time.time()
history3 = model1a_multi.fit(
    X_train, y_train,
    validation_data=(X_test, y_test),
    epochs=20,
    batch_size=128
)

end_time = time.time()
time3 = end_time - start_time
print(f"\nTraining time: {time3:.2f} seconds")

Epoch 1/20
13/13 ━━━━━━━━━━━━━━━━━━━━ 34s 2s/step - accuracy: 0.1474 - auc: 0.5388 - loss: 490.2636 - precision: 0.1479 - recall: 0.1436 - val_accuracy: 0.0227 - val_auc: 0.5114 - val_loss: 2.4803 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00
Epoch 2/20
13/13 ━━━━━━━━━━━━━━━━━━━━ 33s 3s/step - accuracy: 0.1077 - auc: 0.5389 - loss: 4.6898 - precision: 0.1610 - recall: 0.0580 - val_accuracy: 0.1511 - val_auc: 0.5459 - val_loss: 2.4739 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00
Epoch 3/20
13/13 ━━━━━━━━━━━━━━━━━━━━ 33s 3s/step - accuracy: 0.1850 - auc: 0.5440 - loss: 2.4723 - precision: 0.0000e+00 - recall: 0.0000e+00 - val_accuracy: 0.3174 - val_auc: 0.5255 - val_loss: 2.4738 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00
Epoch 4/20
13/13 ━━━━━━━━━━━━━━━━━━━━ 33s 3s/step - accuracy: 0.3251 - auc: 0.6218 - loss: 2.4475 - precision: 0.2931 - recall: 0.0156 - val_accuracy: 0.3325 - val_auc: 0.6755 - val_loss: 2.4626 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00
Epoch 5/20
13/13 ━━━━━━━━━━━━━━━━━━━━ 34s 3s/step - accuracy: 0.3568 - auc: 0.7039 - loss: 2.4295 - precision: 0.2080 - recall: 0.0055 - val_accuracy: 0.3073 - val_auc: 0.7320 - val_loss: 2.3537 - val_precision: 0.8000 - val_recall: 0.0101
Epoch 6/20
13/13 ━━━━━━━━━━━━━━━━━━━━ 33s 3s/step - accuracy: 0.3726 - auc: 0.7713 - loss: 2.2719 - precision: 0.9558 - recall: 0.0228 - val_accuracy: 0.3627 - val_auc: 0.7614 - val_loss: 2.1646 - val_precision: 0.2973 - val_recall: 0.0277
Epoch 7/20
13/13 ━━━━━━━━━━━━━━━━━━━━ 32s 2s/step - accuracy: 0.4831 - auc: 0.8300 - loss: 1.8891 - precision: 0.6481 - recall: 0.1299 - val_accuracy: 0.4232 - val_auc: 0.7944 - val_loss: 2.0609 - val_precision: 0.5044 - val_recall: 0.1436
Epoch 8/20
13/13 ━━━━━━━━━━━━━━━━━━━━ 32s 2s/step - accuracy: 0.5993 - auc: 0.9144 - loss: 1.3373 - precision: 0.7948 - recall: 0.3693 - val_accuracy: 0.3904 - val_auc: 0.7869 - val_loss: 2.1221 - val_precision: 0.4809 - val_recall: 0.1587
Epoch 9/20
13/13 ━━━━━━━━━━━━━━━━━━━━ 32s 2s/step - accuracy: 0.7471 - auc: 0.9612 - loss: 0.9261 - precision: 0.8851 - recall: 0.5404 - val_accuracy: 0.3678 - val_auc: 0.7654 - val_loss: 2.3917 - val_precision: 0.4556 - val_recall: 0.2065
Epoch 10/20
13/13 ━━━━━━━━━━━━━━━━━━━━ 33s 3s/step - accuracy: 0.8065 - auc: 0.9766 - loss: 0.7406 - precision: 0.9126 - recall: 0.6287 - val_accuracy: 0.4181 - val_auc: 0.7855 - val_loss: 2.6065 - val_precision: 0.4743 - val_recall: 0.3023
Epoch 11/20
13/13 ━━━━━━━━━━━━━━━━━━━━ 34s 3s/step - accuracy: 0.8809 - auc: 0.9901 - loss: 0.4354 - precision: 0.9246 - recall: 0.7986 - val_accuracy: 0.3955 - val_auc: 0.7822 - val_loss: 2.6364 - val_precision: 0.4563 - val_recall: 0.2897
Epoch 12/20
13/13 ━━━━━━━━━━━━━━━━━━━━ 33s 3s/step - accuracy: 0.9286 - auc: 0.9970 - loss: 0.2632 - precision: 0.9641 - recall: 0.8865 - val_accuracy: 0.3955 - val_auc: 0.7760 - val_loss: 2.9961 - val_precision: 0.4621 - val_recall: 0.3375
Epoch 13/20
13/13 ━━━━━━━━━━━━━━━━━━━━ 31s 2s/step - accuracy: 0.9571 - auc: 0.9989 - loss: 0.1614 - precision: 0.9749 - recall: 0.9297 - val_accuracy: 0.3955 - val_auc: 0.7766 - val_loss: 3.1419 - val_precision: 0.4400 - val_recall: 0.3325
Epoch 14/20
13/13 ━━━━━━━━━━━━━━━━━━━━ 32s 2s/step - accuracy: 0.9781 - auc: 0.9994 - loss: 0.1087 - precision: 0.9875 - recall: 0.9576 - val_accuracy: 0.3980 - val_auc: 0.7760 - val_loss: 3.2108 - val_precision: 0.4295 - val_recall: 0.3375
Epoch 15/20
13/13 ━━━━━━━━━━━━━━━━━━━━ 34s 3s/step - accuracy: 0.9758 - auc: 0.9994 - loss: 0.0937 - precision: 0.9864 - recall: 0.9628 - val_accuracy: 0.4106 - val_auc: 0.7697 - val_loss: 3.4532 - val_precision: 0.4303 - val_recall: 0.3501
Epoch 16/20
13/13 ━━━━━━━━━━━━━━━━━━━━ 31s 2s/step - accuracy: 0.9811 - auc: 0.9996 - loss: 0.0762 - precision: 0.9842 - recall: 0.9718 - val_accuracy: 0.3854 - val_auc: 0.7689 - val_loss: 3.5760 - val_precision: 0.4149 - val_recall: 0.3375
Epoch 17/20
13/13 ━━━━━━━━━━━━━━━━━━━━ 32s 3s/step - accuracy: 0.9813 - auc: 0.9991 - loss: 0.0869 - precision: 0.9856 - recall: 0.9717 - val_accuracy: 0.3929 - val_auc: 0.7609 - val_loss: 3.6187 - val_precision: 0.4182 - val_recall: 0.3350
Epoch 18/20
13/13 ━━━━━━━━━━━━━━━━━━━━ 34s 3s/step - accuracy: 0.9829 - auc: 0.9993 - loss: 0.0803 - precision: 0.9859 - recall: 0.9692 - val_accuracy: 0.3829 - val_auc: 0.7631 - val_loss: 3.6721 - val_precision: 0.4322 - val_recall: 0.3451
Epoch 19/20
13/13 ━━━━━━━━━━━━━━━━━━━━ 34s 3s/step - accuracy: 0.9850 - auc: 0.9996 - loss: 0.0590 - precision: 0.9900 - recall: 0.9770 - val_accuracy: 0.3879 - val_auc: 0.7549 - val_loss: 3.8591 - val_precision: 0.4218 - val_recall: 0.3602
Epoch 20/20
13/13 ━━━━━━━━━━━━━━━━━━━━ 33s 3s/step - accuracy: 0.9867 - auc: 0.9999 - loss: 0.0456 - precision: 0.9921 - recall: 0.9839 - val_accuracy: 0.4055 - val_auc: 0.7673 - val_loss: 3.8660 - val_precision: 0.4332 - val_recall: 0.3678

Training time: 659.30 seconds
In [20]:
print("\nValidation Accuracy Model3:")
for m in ['val_accuracy', 'val_precision', 'val_recall', 'val_auc']:
    print(f"{m}: {history3.history[m][-1]:.4f}")

Validation Accuracy Model3:
val_accuracy: 0.4055
val_precision: 0.4332
val_recall: 0.3678
val_auc: 0.7673
In [21]:
plt.figure(figsize=(12, 5))

plt.subplot(1, 2, 1)
plt.plot(history3.history['accuracy'], label='Train Accuracy')
plt.plot(history3.history['val_accuracy'], label='Val Accuracy')
plt.title('Model Accuracy')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.legend()

plt.subplot(1, 2, 2)
plt.plot(history3.history['loss'], label='Train Loss')
plt.plot(history3.history['val_loss'], label='Val Loss')
plt.title('Model Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()

plt.tight_layout()
plt.show()
No description has been provided for this image
In [22]:
y_pred = model1a_multi.predict(X_test)
y_pred_classes = np.argmax(y_pred, axis=1)
y_true = np.argmax(y_test, axis=1)

cm3 = confusion_matrix(y_true, y_pred_classes)
plt.figure(figsize=(8, 6))
sns.heatmap(cm3, annot=True, fmt='d', cmap='Blues',
            xticklabels=label_encoder.classes_,
            yticklabels=label_encoder.classes_)

plt.xlabel('Predicted')
plt.ylabel('True')
plt.title('Confusion Matrix Heatmap')
plt.xticks(rotation=90)
plt.yticks(rotation=0)
plt.tight_layout()
plt.show()

13/13 ━━━━━━━━━━━━━━━━━━━━ 2s 125ms/step
No description has been provided for this image
Model1-B¶
In [23]:
np.random.seed(160)
tf.random.set_seed(160)

model1b_multi = Sequential()
model1b_multi.add(Conv2D(32, (3, 3), activation='relu', input_shape=(128, 517, 1)))
model1b_multi.add(MaxPooling2D(pool_size=(2, 2)))

model1b_multi.add(Conv2D(64, (3, 3), activation='relu'))
model1b_multi.add(MaxPooling2D(pool_size=(2, 2)))

model1b_multi.add(Flatten())
model1b_multi.add(Dense(128, activation='relu'))
model1b_multi.add(Dropout(0.3))

model1b_multi.add(Dense(12, activation='softmax'))  

model1b_multi.compile(
    optimizer='adam',
    loss='categorical_crossentropy',
    metrics=[
        'accuracy',
        Precision(name='precision'),
        Recall(name='recall'),
        AUC(name='auc')
    ]
)

C:\Users\Nidhi\anaconda3\envs\py311\Lib\site-packages\keras\src\layers\convolutional\base_conv.py:107: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)
In [24]:
start_time = time.time()

history4= model1b_multi.fit(
    X_train, y_train,
    epochs=15,
    batch_size=64,
    validation_data=(X_test, y_test)
)

end_time = time.time()
time4 = end_time - start_time

print(f"Training time: {time4:.2f} seconds ({time4 / 60:.2f} minutes)")

Epoch 1/15
25/25 ━━━━━━━━━━━━━━━━━━━━ 37s 1s/step - accuracy: 0.1688 - auc: 0.5577 - loss: 277.2318 - precision: 0.1606 - recall: 0.1203 - val_accuracy: 0.3652 - val_auc: 0.7518 - val_loss: 2.2955 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00
Epoch 2/15
25/25 ━━━━━━━━━━━━━━━━━━━━ 34s 1s/step - accuracy: 0.3851 - auc: 0.7703 - loss: 2.1832 - precision: 0.5867 - recall: 0.0304 - val_accuracy: 0.3955 - val_auc: 0.7954 - val_loss: 2.0291 - val_precision: 0.5455 - val_recall: 0.0453
Epoch 3/15
25/25 ━━━━━━━━━━━━━━━━━━━━ 33s 1s/step - accuracy: 0.5282 - auc: 0.8698 - loss: 1.6901 - precision: 0.7639 - recall: 0.1532 - val_accuracy: 0.4257 - val_auc: 0.8044 - val_loss: 1.9602 - val_precision: 0.5440 - val_recall: 0.1713
Epoch 4/15
25/25 ━━━━━━━━━━━━━━━━━━━━ 33s 1s/step - accuracy: 0.6821 - auc: 0.9448 - loss: 1.1306 - precision: 0.8553 - recall: 0.4145 - val_accuracy: 0.3678 - val_auc: 0.8055 - val_loss: 2.2662 - val_precision: 0.4735 - val_recall: 0.2695
Epoch 5/15
25/25 ━━━━━━━━━━━━━━━━━━━━ 33s 1s/step - accuracy: 0.7801 - auc: 0.9670 - loss: 0.8575 - precision: 0.8845 - recall: 0.5553 - val_accuracy: 0.4433 - val_auc: 0.8014 - val_loss: 2.3811 - val_precision: 0.5075 - val_recall: 0.3401
Epoch 6/15
25/25 ━━━━━━━━━━━━━━━━━━━━ 33s 1s/step - accuracy: 0.8629 - auc: 0.9911 - loss: 0.4389 - precision: 0.9223 - recall: 0.7953 - val_accuracy: 0.4307 - val_auc: 0.7871 - val_loss: 2.8283 - val_precision: 0.4704 - val_recall: 0.3602
Epoch 7/15
25/25 ━━━━━━━━━━━━━━━━━━━━ 35s 1s/step - accuracy: 0.9368 - auc: 0.9979 - loss: 0.2201 - precision: 0.9600 - recall: 0.8997 - val_accuracy: 0.4358 - val_auc: 0.7741 - val_loss: 3.3975 - val_precision: 0.4688 - val_recall: 0.3980
Epoch 8/15
25/25 ━━━━━━━━━━━━━━━━━━━━ 35s 1s/step - accuracy: 0.9330 - auc: 0.9913 - loss: 0.3123 - precision: 0.9470 - recall: 0.9139 - val_accuracy: 0.4030 - val_auc: 0.7788 - val_loss: 2.8058 - val_precision: 0.4586 - val_recall: 0.3350
Epoch 9/15
25/25 ━━━━━━━━━━━━━━━━━━━━ 33s 1s/step - accuracy: 0.9369 - auc: 0.9922 - loss: 0.2830 - precision: 0.9527 - recall: 0.8920 - val_accuracy: 0.4559 - val_auc: 0.7755 - val_loss: 3.2140 - val_precision: 0.5061 - val_recall: 0.4207
Epoch 10/15
25/25 ━━━━━━━━━━━━━━━━━━━━ 33s 1s/step - accuracy: 0.9678 - auc: 0.9993 - loss: 0.1325 - precision: 0.9752 - recall: 0.9502 - val_accuracy: 0.4282 - val_auc: 0.7860 - val_loss: 3.2643 - val_precision: 0.4679 - val_recall: 0.3854
Epoch 11/15
25/25 ━━━━━━━━━━━━━━━━━━━━ 33s 1s/step - accuracy: 0.9704 - auc: 0.9995 - loss: 0.1152 - precision: 0.9789 - recall: 0.9520 - val_accuracy: 0.4660 - val_auc: 0.7932 - val_loss: 3.1306 - val_precision: 0.4882 - val_recall: 0.4156
Epoch 12/15
25/25 ━━━━━━━━━━━━━━━━━━━━ 33s 1s/step - accuracy: 0.9834 - auc: 0.9991 - loss: 0.0644 - precision: 0.9870 - recall: 0.9752 - val_accuracy: 0.4232 - val_auc: 0.7987 - val_loss: 3.3640 - val_precision: 0.4599 - val_recall: 0.3904
Epoch 13/15
25/25 ━━━━━━━━━━━━━━━━━━━━ 34s 1s/step - accuracy: 0.9837 - auc: 0.9998 - loss: 0.0568 - precision: 0.9879 - recall: 0.9802 - val_accuracy: 0.4358 - val_auc: 0.7812 - val_loss: 3.7600 - val_precision: 0.4718 - val_recall: 0.4207
Epoch 14/15
25/25 ━━━━━━━━━━━━━━━━━━━━ 33s 1s/step - accuracy: 0.9933 - auc: 1.0000 - loss: 0.0276 - precision: 0.9953 - recall: 0.9916 - val_accuracy: 0.4332 - val_auc: 0.7918 - val_loss: 3.4574 - val_precision: 0.4687 - val_recall: 0.3955
Epoch 15/15
25/25 ━━━━━━━━━━━━━━━━━━━━ 33s 1s/step - accuracy: 0.9900 - auc: 0.9999 - loss: 0.0329 - precision: 0.9932 - recall: 0.9883 - val_accuracy: 0.4484 - val_auc: 0.7943 - val_loss: 3.4947 - val_precision: 0.4926 - val_recall: 0.4207
Training time: 506.91 seconds (8.45 minutes)
In [25]:
print("\nValidation Accuracy Model4:")
for m in ['val_accuracy', 'val_precision', 'val_recall', 'val_auc']:
    print(f"{m}: {history4.history[m][-1]:.4f}")

Validation Accuracy Model4:
val_accuracy: 0.4484
val_precision: 0.4926
val_recall: 0.4207
val_auc: 0.7943
In [26]:
plt.figure(figsize=(12, 5))

plt.subplot(1, 2, 1)
plt.plot(history4.history['accuracy'], label='Train Accuracy')
plt.plot(history4.history['val_accuracy'], label='Val Accuracy')
plt.title('Model Accuracy')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.legend()

plt.subplot(1, 2, 2)
plt.plot(history4.history['loss'], label='Train Loss')
plt.plot(history4.history['val_loss'], label='Val Loss')
plt.title('Model Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()

plt.tight_layout()
plt.show()
No description has been provided for this image
In [27]:
y_pred = model1b_multi.predict(X_test)
y_pred_classes = np.argmax(y_pred, axis=1)
y_true = np.argmax(y_test, axis=1)

cm4 = confusion_matrix(y_true, y_pred_classes)
plt.figure(figsize=(8, 6))
sns.heatmap(cm4, annot=True, fmt='d', cmap='Blues',
            xticklabels=label_encoder.classes_,
            yticklabels=label_encoder.classes_)

plt.xlabel('Predicted')
plt.ylabel('True')
plt.title('Confusion Matrix Heatmap')
plt.xticks(rotation=90)
plt.yticks(rotation=0)
plt.tight_layout()
plt.show()

13/13 ━━━━━━━━━━━━━━━━━━━━ 2s 121ms/step
No description has been provided for this image

Model2-A¶

In [28]:
np.random.seed(160)
tf.random.set_seed(160)


model2a_multi = Sequential()
model2a_multi.add(Conv2D(16, (3, 3), activation='relu', input_shape=(128, 517, 1)))
model2a_multi.add(MaxPooling2D(pool_size=(2, 2)))

model2a_multi.add(Conv2D(32, (3, 3), activation='relu'))
model2a_multi.add(MaxPooling2D(pool_size=(2, 2)))

model2a_multi.add(Flatten())
model2a_multi.add(Dense(64, activation='relu'))
model2a_multi.add(Dropout(0.4))

model2a_multi.add(Dense(12, activation='softmax'))  

model2a_multi.compile(
    optimizer='adam',
    loss='categorical_crossentropy',
    metrics=[
        'accuracy',
        Precision(name='precision'),
        Recall(name='recall'),
        AUC(name='auc')
    ]
)

model2a_multi.summary()

C:\Users\Nidhi\anaconda3\envs\py311\Lib\site-packages\keras\src\layers\convolutional\base_conv.py:107: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)

Model: "sequential_4"

┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓
┃ Layer (type)                    ┃ Output Shape           ┃       Param # ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩
│ conv2d_8 (Conv2D)               │ (None, 126, 515, 16)   │           160 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ max_pooling2d_8 (MaxPooling2D)  │ (None, 63, 257, 16)    │             0 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ conv2d_9 (Conv2D)               │ (None, 61, 255, 32)    │         4,640 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ max_pooling2d_9 (MaxPooling2D)  │ (None, 30, 127, 32)    │             0 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ flatten_4 (Flatten)             │ (None, 121920)         │             0 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ dense_8 (Dense)                 │ (None, 64)             │     7,802,944 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ dropout_4 (Dropout)             │ (None, 64)             │             0 │
├─────────────────────────────────┼────────────────────────┼───────────────┤
│ dense_9 (Dense)                 │ (None, 12)             │           780 │
└─────────────────────────────────┴────────────────────────┴───────────────┘

 Total params: 7,808,524 (29.79 MB)

 Trainable params: 7,808,524 (29.79 MB)

 Non-trainable params: 0 (0.00 B)
In [29]:
start_time = time.time()
history6= model2a_multi.fit(
    X_train, y_train,
    epochs=15,
    batch_size=32,
    validation_data=(X_test, y_test)
)

end_time = time.time()
time6 = end_time - start_time

print(f"Training time: {time6:.2f} seconds ({time6 / 60:.2f} minutes)")

Epoch 1/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 13s 218ms/step - accuracy: 0.1576 - auc: 0.5262 - loss: 89.9453 - precision: 0.1160 - recall: 0.0574 - val_accuracy: 0.2796 - val_auc: 0.6792 - val_loss: 2.3146 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00
Epoch 2/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 11s 211ms/step - accuracy: 0.2761 - auc: 0.6786 - loss: 2.3032 - precision: 0.6057 - recall: 0.0140 - val_accuracy: 0.3325 - val_auc: 0.7132 - val_loss: 2.1947 - val_precision: 0.5946 - val_recall: 0.0554
Epoch 3/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 11s 210ms/step - accuracy: 0.3382 - auc: 0.7604 - loss: 2.0945 - precision: 0.7583 - recall: 0.0832 - val_accuracy: 0.3804 - val_auc: 0.7514 - val_loss: 2.1184 - val_precision: 0.7500 - val_recall: 0.0378
Epoch 4/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 11s 213ms/step - accuracy: 0.4139 - auc: 0.8156 - loss: 1.8752 - precision: 0.7625 - recall: 0.1354 - val_accuracy: 0.3804 - val_auc: 0.7733 - val_loss: 2.0503 - val_precision: 0.6207 - val_recall: 0.0453
Epoch 5/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 11s 213ms/step - accuracy: 0.4658 - auc: 0.8616 - loss: 1.6769 - precision: 0.8261 - recall: 0.2101 - val_accuracy: 0.4005 - val_auc: 0.7823 - val_loss: 2.0135 - val_precision: 0.6559 - val_recall: 0.1537
Epoch 6/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 11s 213ms/step - accuracy: 0.4949 - auc: 0.9019 - loss: 1.4338 - precision: 0.8300 - recall: 0.3012 - val_accuracy: 0.4156 - val_auc: 0.7857 - val_loss: 2.0423 - val_precision: 0.6354 - val_recall: 0.1537
Epoch 7/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 11s 212ms/step - accuracy: 0.5460 - auc: 0.9238 - loss: 1.2751 - precision: 0.8316 - recall: 0.3572 - val_accuracy: 0.4257 - val_auc: 0.7920 - val_loss: 2.0858 - val_precision: 0.6071 - val_recall: 0.2569
Epoch 8/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 11s 211ms/step - accuracy: 0.5891 - auc: 0.9471 - loss: 1.0776 - precision: 0.8539 - recall: 0.4484 - val_accuracy: 0.4181 - val_auc: 0.7821 - val_loss: 2.2802 - val_precision: 0.5674 - val_recall: 0.3073
Epoch 9/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 11s 216ms/step - accuracy: 0.6204 - auc: 0.9566 - loss: 0.9760 - precision: 0.8708 - recall: 0.4972 - val_accuracy: 0.4055 - val_auc: 0.7879 - val_loss: 2.2989 - val_precision: 0.5388 - val_recall: 0.2972
Epoch 10/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 11s 214ms/step - accuracy: 0.6574 - auc: 0.9627 - loss: 0.8983 - precision: 0.8973 - recall: 0.5350 - val_accuracy: 0.4156 - val_auc: 0.7896 - val_loss: 2.2420 - val_precision: 0.5803 - val_recall: 0.2821
Epoch 11/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 11s 215ms/step - accuracy: 0.6881 - auc: 0.9693 - loss: 0.8198 - precision: 0.8711 - recall: 0.5819 - val_accuracy: 0.4232 - val_auc: 0.7922 - val_loss: 2.4287 - val_precision: 0.5569 - val_recall: 0.3451
Epoch 12/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 11s 215ms/step - accuracy: 0.7021 - auc: 0.9742 - loss: 0.7545 - precision: 0.8710 - recall: 0.6122 - val_accuracy: 0.4332 - val_auc: 0.7873 - val_loss: 2.4890 - val_precision: 0.5542 - val_recall: 0.3350
Epoch 13/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 11s 214ms/step - accuracy: 0.7285 - auc: 0.9751 - loss: 0.7277 - precision: 0.8831 - recall: 0.6360 - val_accuracy: 0.4181 - val_auc: 0.7833 - val_loss: 2.4983 - val_precision: 0.5391 - val_recall: 0.3300
Epoch 14/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 11s 219ms/step - accuracy: 0.7612 - auc: 0.9800 - loss: 0.6484 - precision: 0.8878 - recall: 0.6555 - val_accuracy: 0.4257 - val_auc: 0.7851 - val_loss: 2.5617 - val_precision: 0.5245 - val_recall: 0.3501
Epoch 15/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 11s 215ms/step - accuracy: 0.7615 - auc: 0.9785 - loss: 0.6538 - precision: 0.8703 - recall: 0.6647 - val_accuracy: 0.4131 - val_auc: 0.7976 - val_loss: 2.3840 - val_precision: 0.5370 - val_recall: 0.2922
Training time: 162.97 seconds (2.72 minutes)
In [30]:
print("\nValidation Accuracy Model6:")
for m in ['val_accuracy', 'val_precision', 'val_recall', 'val_auc']:
    print(f"{m}: {history6.history[m][-1]:.4f}")

Validation Accuracy Model6:
val_accuracy: 0.4131
val_precision: 0.5370
val_recall: 0.2922
val_auc: 0.7976
In [31]:
plt.figure(figsize=(12, 5))

plt.subplot(1, 2, 1)
plt.plot(history6.history['accuracy'], label='Train Accuracy')
plt.plot(history6.history['val_accuracy'], label='Val Accuracy')
plt.title('Model Accuracy')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.legend()

plt.subplot(1, 2, 2)
plt.plot(history6.history['loss'], label='Train Loss')
plt.plot(history6.history['val_loss'], label='Val Loss')
plt.title('Model Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()

plt.tight_layout()
plt.show()
No description has been provided for this image
In [32]:
y_pred = model2a_multi.predict(X_test)
y_pred_classes = np.argmax(y_pred, axis=1)
y_true = np.argmax(y_test, axis=1)

cm6 = confusion_matrix(y_true, y_pred_classes)
plt.figure(figsize=(8, 6))
sns.heatmap(cm6, annot=True, fmt='d', cmap='Blues',
            xticklabels=label_encoder.classes_,
            yticklabels=label_encoder.classes_)

plt.xlabel('Predicted')
plt.ylabel('True')
plt.title('Confusion Matrix Heatmap')
plt.xticks(rotation=90)
plt.yticks(rotation=0)
plt.tight_layout()
plt.show()

13/13 ━━━━━━━━━━━━━━━━━━━━ 1s 51ms/step
No description has been provided for this image

Addressing class imbalance issue¶

In [ ]:
* As we can see by tuning both cnn convolutional layers with different dropout rates and batches, still model is highly favoring housesparrow as it has more samples in dataset.
* To address these issue of class imbalance let's use class weight for penalizing wrong prediction og lower classes samples using class weights.
* The class weights were used on  models which achieved good accuracy in previous model to see if it affects accuracy and prediction.
* Models used are model1b and model2 with class weights.
In [7]:
from sklearn.utils import class_weight
class_weights = class_weight.compute_class_weight(
    class_weight='balanced',
    classes=np.unique(y_encoded),
    y=y_encoded
)

# Convert to dictionary
class_weights_dict = dict(enumerate(class_weights))
In [13]:
np.random.seed(160)
tf.random.set_seed(160)

model1b_weights = Sequential()
model1b_weights.add(Conv2D(32, (3, 3), activation='relu', input_shape=(128, 517, 1)))
model1b_weights.add(MaxPooling2D(pool_size=(2, 2)))

model1b_weights.add(Conv2D(64, (3, 3), activation='relu'))
model1b_weights.add(MaxPooling2D(pool_size=(2, 2)))

model1b_weights.add(Flatten())
model1b_weights.add(Dense(128, activation='relu'))
model1b_weights.add(Dropout(0.3))
model1b_weights.add(Dense(12, activation='softmax'))  

model1b_weights.compile(
    optimizer='adam',
    loss='categorical_crossentropy',
    metrics=[
        'accuracy',
        Precision(name='precision'),
        Recall(name='recall'),
        AUC(name='auc')
    ]
)
In [14]:
import time
start_time = time.time()
history7= model1b_weights.fit(
    X_train, y_train,
    epochs=15,
    batch_size=32,
    validation_data=(X_test, y_test),
    class_weight=class_weights_dict
)

end_time = time.time()
time7 = end_time - start_time

print(f"Training time: {time7:.2f} seconds ({time7 / 60:.2f} minutes)")

Epoch 1/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 40s 772ms/step - accuracy: 0.1300 - auc: 0.5399 - loss: 153.5328 - precision: 0.0898 - recall: 0.0462 - val_accuracy: 0.2065 - val_auc: 0.6707 - val_loss: 2.3228 - val_precision: 0.5000 - val_recall: 0.0076
Epoch 2/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 40s 800ms/step - accuracy: 0.3086 - auc: 0.7453 - loss: 2.2780 - precision: 0.7043 - recall: 0.0409 - val_accuracy: 0.2469 - val_auc: 0.7207 - val_loss: 2.2517 - val_precision: 0.6034 - val_recall: 0.0882
Epoch 3/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 40s 793ms/step - accuracy: 0.4634 - auc: 0.8686 - loss: 1.5347 - precision: 0.7632 - recall: 0.2183 - val_accuracy: 0.2494 - val_auc: 0.7097 - val_loss: 2.5521 - val_precision: 0.4225 - val_recall: 0.1511
Epoch 4/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 39s 778ms/step - accuracy: 0.6643 - auc: 0.9452 - loss: 0.8585 - precision: 0.8492 - recall: 0.4863 - val_accuracy: 0.2469 - val_auc: 0.7254 - val_loss: 2.3152 - val_precision: 0.4337 - val_recall: 0.0907
Epoch 5/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 40s 800ms/step - accuracy: 0.7698 - auc: 0.9671 - loss: 0.6320 - precision: 0.8951 - recall: 0.6179 - val_accuracy: 0.2947 - val_auc: 0.7466 - val_loss: 2.6416 - val_precision: 0.4144 - val_recall: 0.1889
Epoch 6/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 39s 780ms/step - accuracy: 0.8514 - auc: 0.9869 - loss: 0.3443 - precision: 0.9285 - recall: 0.7753 - val_accuracy: 0.3199 - val_auc: 0.7573 - val_loss: 2.9091 - val_precision: 0.3852 - val_recall: 0.2368
Epoch 7/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 39s 773ms/step - accuracy: 0.9152 - auc: 0.9955 - loss: 0.1932 - precision: 0.9607 - recall: 0.8570 - val_accuracy: 0.3602 - val_auc: 0.7483 - val_loss: 3.9328 - val_precision: 0.4013 - val_recall: 0.3174
Epoch 8/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 39s 778ms/step - accuracy: 0.9557 - auc: 0.9980 - loss: 0.1048 - precision: 0.9773 - recall: 0.9271 - val_accuracy: 0.3552 - val_auc: 0.7391 - val_loss: 3.9011 - val_precision: 0.4082 - val_recall: 0.3249
Epoch 9/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 40s 796ms/step - accuracy: 0.9765 - auc: 0.9993 - loss: 0.0628 - precision: 0.9866 - recall: 0.9598 - val_accuracy: 0.3577 - val_auc: 0.7346 - val_loss: 3.8899 - val_precision: 0.3820 - val_recall: 0.3098
Epoch 10/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 39s 776ms/step - accuracy: 0.9785 - auc: 0.9997 - loss: 0.0487 - precision: 0.9844 - recall: 0.9658 - val_accuracy: 0.3401 - val_auc: 0.7331 - val_loss: 3.9741 - val_precision: 0.3795 - val_recall: 0.3174
Epoch 11/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 38s 757ms/step - accuracy: 0.9810 - auc: 0.9993 - loss: 0.0475 - precision: 0.9905 - recall: 0.9680 - val_accuracy: 0.3602 - val_auc: 0.7371 - val_loss: 4.3465 - val_precision: 0.3964 - val_recall: 0.3325
Epoch 12/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 38s 769ms/step - accuracy: 0.9887 - auc: 0.9999 - loss: 0.0431 - precision: 0.9944 - recall: 0.9825 - val_accuracy: 0.3627 - val_auc: 0.7314 - val_loss: 4.3610 - val_precision: 0.3897 - val_recall: 0.3249
Epoch 13/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 40s 798ms/step - accuracy: 0.9886 - auc: 0.9998 - loss: 0.0460 - precision: 0.9905 - recall: 0.9807 - val_accuracy: 0.3627 - val_auc: 0.7316 - val_loss: 4.6658 - val_precision: 0.3848 - val_recall: 0.3451
Epoch 14/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 40s 792ms/step - accuracy: 0.9766 - auc: 0.9985 - loss: 0.0901 - precision: 0.9808 - recall: 0.9684 - val_accuracy: 0.3678 - val_auc: 0.7316 - val_loss: 4.6920 - val_precision: 0.4000 - val_recall: 0.3375
Epoch 15/15
50/50 ━━━━━━━━━━━━━━━━━━━━ 40s 804ms/step - accuracy: 0.9869 - auc: 0.9993 - loss: 0.0532 - precision: 0.9921 - recall: 0.9751 - val_accuracy: 0.3401 - val_auc: 0.7209 - val_loss: 4.5728 - val_precision: 0.3754 - val_recall: 0.3073
Training time: 590.58 seconds (9.84 minutes)
In [16]:
print("\nValidation Accuracy Model7")
for m in ['val_accuracy', 'val_precision', 'val_recall', 'val_auc']:
    print(f"{m}: {history7.history[m][-1]:.4f}")

Validation Accuracy Model7
val_accuracy: 0.3401
val_precision: 0.3754
val_recall: 0.3073
val_auc: 0.7209
In [17]:
plt.figure(figsize=(12, 5))

plt.subplot(1, 2, 1)
plt.plot(history7.history['accuracy'], label='Train Accuracy')
plt.plot(history7.history['val_accuracy'], label='Val Accuracy')
plt.title('Model Accuracy')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.legend()

plt.subplot(1, 2, 2)
plt.plot(history7.history['loss'], label='Train Loss')
plt.plot(history7.history['val_loss'], label='Val Loss')
plt.title('Model Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()

plt.tight_layout()
plt.show()
No description has been provided for this image
In [19]:
from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay
import matplotlib.pyplot as plt
y_pred = model1b_weights.predict(X_test)
y_pred_classes = np.argmax(y_pred, axis=1)
y_true = np.argmax(y_test, axis=1)

cm7 = confusion_matrix(y_true, y_pred_classes)
plt.figure(figsize=(8, 6))
sns.heatmap(cm7, annot=True, fmt='d', cmap='Blues',
            xticklabels=label_encoder.classes_,
            yticklabels=label_encoder.classes_)

plt.xlabel('Predicted')
plt.ylabel('True')
plt.title('Confusion Matrix Heatmap')
plt.xticks(rotation=90)
plt.yticks(rotation=0)
plt.tight_layout()
plt.show()

13/13 ━━━━━━━━━━━━━━━━━━━━ 2s 114ms/step
No description has been provided for this image
In [34]:
# Defining model2
np.random.seed(42)
tf.random.set_seed(42)


model2_weights = Sequential()
model2_weights.add(Conv2D(16, (3, 3), activation='relu', input_shape=(128, 517, 1)))
model2_weights.add(MaxPooling2D(pool_size=(2, 2)))

model2_weights.add(Conv2D(32, (3, 3), activation='relu'))
model2_weights.add(MaxPooling2D(pool_size=(2, 2)))

model2_weights.add(Flatten())
model2_weights.add(Dense(64, activation='relu'))
model2_weights.add(Dropout(0.3))

model2_weights.add(Dense(12, activation='softmax'))  

model2_weights.compile(
    optimizer='adam',
    loss='categorical_crossentropy',
    metrics=[
        'accuracy',
        Precision(name='precision'),
        Recall(name='recall'),
        AUC(name='auc')
    ]
)

C:\Users\Nidhi\anaconda3\envs\py311\Lib\site-packages\keras\src\layers\convolutional\base_conv.py:107: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)
In [35]:
import time
start_time = time.time()
history9=model2_weights.fit(
    X_train, y_train,
    epochs=25,
    batch_size=128,
    validation_data=(X_test, y_test),
    class_weight=class_weights_dict
)

end_time = time.time()
time8 = end_time - start_time

print(f"Training time: {time8:.2f} seconds ({time8 / 60:.2f} minutes)")

Epoch 1/25
13/13 ━━━━━━━━━━━━━━━━━━━━ 11s 690ms/step - accuracy: 0.0689 - auc: 0.4961 - loss: 163.2697 - precision: 0.0691 - recall: 0.0642 - val_accuracy: 0.0428 - val_auc: 0.4999 - val_loss: 2.4839 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00
Epoch 2/25
13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 672ms/step - accuracy: 0.0508 - auc: 0.5041 - loss: 2.5163 - precision: 0.0000e+00 - recall: 0.0000e+00 - val_accuracy: 0.2569 - val_auc: 0.5209 - val_loss: 2.4785 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00
Epoch 3/25
13/13 ━━━━━━━━━━━━━━━━━━━━ 8s 640ms/step - accuracy: 0.1462 - auc: 0.5263 - loss: 2.5020 - precision: 0.0000e+00 - recall: 0.0000e+00 - val_accuracy: 0.0453 - val_auc: 0.5191 - val_loss: 2.4797 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00
Epoch 4/25
13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 653ms/step - accuracy: 0.0734 - auc: 0.5803 - loss: 2.4780 - precision: 0.0000e+00 - recall: 0.0000e+00 - val_accuracy: 0.0781 - val_auc: 0.6296 - val_loss: 2.4560 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00
Epoch 5/25
13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 652ms/step - accuracy: 0.0638 - auc: 0.6437 - loss: 2.4515 - precision: 0.0000e+00 - recall: 0.0000e+00 - val_accuracy: 0.0856 - val_auc: 0.6300 - val_loss: 2.4368 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00
Epoch 6/25
13/13 ━━━━━━━━━━━━━━━━━━━━ 8s 648ms/step - accuracy: 0.0890 - auc: 0.6607 - loss: 2.3678 - precision: 0.0000e+00 - recall: 0.0000e+00 - val_accuracy: 0.1134 - val_auc: 0.6442 - val_loss: 2.4103 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00
Epoch 7/25
13/13 ━━━━━━━━━━━━━━━━━━━━ 8s 638ms/step - accuracy: 0.1319 - auc: 0.7008 - loss: 2.2343 - precision: 0.5258 - recall: 0.0028 - val_accuracy: 0.1310 - val_auc: 0.6520 - val_loss: 2.3727 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00
Epoch 8/25
13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 652ms/step - accuracy: 0.1816 - auc: 0.7435 - loss: 2.0882 - precision: 0.7431 - recall: 0.0162 - val_accuracy: 0.1612 - val_auc: 0.6643 - val_loss: 2.3395 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00
Epoch 9/25
13/13 ━━━━━━━━━━━━━━━━━━━━ 8s 630ms/step - accuracy: 0.2214 - auc: 0.7636 - loss: 1.9386 - precision: 0.7876 - recall: 0.0336 - val_accuracy: 0.1940 - val_auc: 0.6988 - val_loss: 2.2705 - val_precision: 0.4444 - val_recall: 0.0101
Epoch 10/25
13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 654ms/step - accuracy: 0.2593 - auc: 0.7997 - loss: 1.7938 - precision: 0.7355 - recall: 0.0707 - val_accuracy: 0.2015 - val_auc: 0.7127 - val_loss: 2.3353 - val_precision: 0.4857 - val_recall: 0.0428
Epoch 11/25
13/13 ━━━━━━━━━━━━━━━━━━━━ 8s 634ms/step - accuracy: 0.3495 - auc: 0.8241 - loss: 1.6483 - precision: 0.7379 - recall: 0.1487 - val_accuracy: 0.1839 - val_auc: 0.6809 - val_loss: 2.4259 - val_precision: 0.3333 - val_recall: 0.0403
Epoch 12/25
13/13 ━━━━━━━━━━━━━━━━━━━━ 8s 633ms/step - accuracy: 0.3874 - auc: 0.8484 - loss: 1.4748 - precision: 0.7241 - recall: 0.1949 - val_accuracy: 0.2519 - val_auc: 0.7337 - val_loss: 2.2310 - val_precision: 0.4000 - val_recall: 0.0403
Epoch 13/25
13/13 ━━━━━━━━━━━━━━━━━━━━ 8s 636ms/step - accuracy: 0.4616 - auc: 0.8888 - loss: 1.2352 - precision: 0.8305 - recall: 0.2657 - val_accuracy: 0.2569 - val_auc: 0.7369 - val_loss: 2.2863 - val_precision: 0.4478 - val_recall: 0.0756
Epoch 14/25
13/13 ━━━━━━━━━━━━━━━━━━━━ 8s 639ms/step - accuracy: 0.5025 - auc: 0.9061 - loss: 1.1478 - precision: 0.8069 - recall: 0.3071 - val_accuracy: 0.2846 - val_auc: 0.7430 - val_loss: 2.2941 - val_precision: 0.4255 - val_recall: 0.1008
Epoch 15/25
13/13 ━━━━━━━━━━━━━━━━━━━━ 8s 630ms/step - accuracy: 0.5384 - auc: 0.9256 - loss: 1.0337 - precision: 0.8381 - recall: 0.3461 - val_accuracy: 0.3048 - val_auc: 0.7619 - val_loss: 2.2328 - val_precision: 0.4949 - val_recall: 0.1234
Epoch 16/25
13/13 ━━━━━━━━━━━━━━━━━━━━ 8s 645ms/step - accuracy: 0.5882 - auc: 0.9414 - loss: 0.9326 - precision: 0.8378 - recall: 0.4139 - val_accuracy: 0.3023 - val_auc: 0.7431 - val_loss: 2.3148 - val_precision: 0.5366 - val_recall: 0.1108
Epoch 17/25
13/13 ━━━━━━━━━━━━━━━━━━━━ 8s 647ms/step - accuracy: 0.6093 - auc: 0.9452 - loss: 0.8621 - precision: 0.9035 - recall: 0.4341 - val_accuracy: 0.3023 - val_auc: 0.7634 - val_loss: 2.4053 - val_precision: 0.4236 - val_recall: 0.1537
Epoch 18/25
13/13 ━━━━━━━━━━━━━━━━━━━━ 8s 631ms/step - accuracy: 0.6514 - auc: 0.9581 - loss: 0.7557 - precision: 0.8517 - recall: 0.5075 - val_accuracy: 0.3224 - val_auc: 0.7524 - val_loss: 2.6215 - val_precision: 0.3795 - val_recall: 0.1587
Epoch 19/25
13/13 ━━━━━━━━━━━━━━━━━━━━ 8s 646ms/step - accuracy: 0.6622 - auc: 0.9629 - loss: 0.7551 - precision: 0.8698 - recall: 0.5303 - val_accuracy: 0.3224 - val_auc: 0.7515 - val_loss: 2.6418 - val_precision: 0.4000 - val_recall: 0.1713
Epoch 20/25
13/13 ━━━━━━━━━━━━━━━━━━━━ 8s 630ms/step - accuracy: 0.6908 - auc: 0.9673 - loss: 0.7316 - precision: 0.8741 - recall: 0.5149 - val_accuracy: 0.3652 - val_auc: 0.7539 - val_loss: 2.9539 - val_precision: 0.4508 - val_recall: 0.2771
Epoch 21/25
13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 653ms/step - accuracy: 0.7017 - auc: 0.9692 - loss: 0.7035 - precision: 0.8686 - recall: 0.5755 - val_accuracy: 0.3325 - val_auc: 0.7609 - val_loss: 2.6510 - val_precision: 0.4375 - val_recall: 0.2116
Epoch 22/25
13/13 ━━━━━━━━━━━━━━━━━━━━ 8s 625ms/step - accuracy: 0.7068 - auc: 0.9697 - loss: 0.6671 - precision: 0.8609 - recall: 0.5762 - val_accuracy: 0.3627 - val_auc: 0.7546 - val_loss: 2.5958 - val_precision: 0.4749 - val_recall: 0.2141
Epoch 23/25
13/13 ━━━━━━━━━━━━━━━━━━━━ 8s 643ms/step - accuracy: 0.7264 - auc: 0.9765 - loss: 0.5769 - precision: 0.8618 - recall: 0.6319 - val_accuracy: 0.3401 - val_auc: 0.7631 - val_loss: 2.5409 - val_precision: 0.4551 - val_recall: 0.2040
Epoch 24/25
13/13 ━━━━━━━━━━━━━━━━━━━━ 8s 640ms/step - accuracy: 0.7524 - auc: 0.9780 - loss: 0.5960 - precision: 0.8717 - recall: 0.6470 - val_accuracy: 0.3123 - val_auc: 0.7560 - val_loss: 2.5722 - val_precision: 0.4362 - val_recall: 0.1637
Epoch 25/25
13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 648ms/step - accuracy: 0.7487 - auc: 0.9775 - loss: 0.5744 - precision: 0.8992 - recall: 0.6270 - val_accuracy: 0.3879 - val_auc: 0.7687 - val_loss: 2.7792 - val_precision: 0.4783 - val_recall: 0.2771
Training time: 213.46 seconds (3.56 minutes)
In [38]:
print("\nValidation Accuracy Model8")
for m in ['val_accuracy', 'val_precision', 'val_recall', 'val_auc']:
    print(f"{m}: {history9.history[m][-1]:.4f}")

Validation Accuracy Model8
val_accuracy: 0.3879
val_precision: 0.4783
val_recall: 0.2771
val_auc: 0.7687
In [36]:
plt.figure(figsize=(12, 5))

plt.subplot(1, 2, 1)
plt.plot(history9.history['accuracy'], label='Train Accuracy')
plt.plot(history9.history['val_accuracy'], label='Val Accuracy')
plt.title('Model Accuracy')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.legend()

plt.subplot(1, 2, 2)
plt.plot(history9.history['loss'], label='Train Loss')
plt.plot(history9.history['val_loss'], label='Val Loss')
plt.title('Model Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()

plt.tight_layout()
plt.show()
No description has been provided for this image
In [37]:
y_pred = model2_weights.predict(X_test)
y_pred_classes = np.argmax(y_pred, axis=1)
y_true = np.argmax(y_test, axis=1)

cm8 = confusion_matrix(y_true, y_pred_classes)
plt.figure(figsize=(8, 6))
sns.heatmap(cm8, annot=True, fmt='d', cmap='Blues',
            xticklabels=label_encoder.classes_,
            yticklabels=label_encoder.classes_)

plt.xlabel('Predicted')
plt.ylabel('True')
plt.title('Confusion Matrix Heatmap')
plt.xticks(rotation=90)
plt.yticks(rotation=0)
plt.tight_layout()
plt.show()

13/13 ━━━━━━━━━━━━━━━━━━━━ 1s 49ms/step
No description has been provided for this image
In [46]:
# saving bestmodel out of all
model1b_multi.save("C:\\Users\\Nidhi\\Documents\\College Material\\Spring Quarter 2025\\SML2\Homework3\\bestmodel1.h5")

WARNING:absl:You are saving your model as an HDF5 file via `model.save()` or `keras.saving.save_model(model)`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')` or `keras.saving.save_model(model, 'my_model.keras')`.
In [47]:
model2_multi.save("C:\\Users\\Nidhi\\Documents\\College Material\\Spring Quarter 2025\\SML2\Homework3\\bestmodel2.h5")

WARNING:absl:You are saving your model as an HDF5 file via `model.save()` or `keras.saving.save_model(model)`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')` or `keras.saving.save_model(model, 'my_model.keras')`.
In [39]:
model2_weights.save("C:\\Users\\Nidhi\\Documents\\College Material\\Spring Quarter 2025\\SML2\Homework3\\bestmodel3.h5")

WARNING:absl:You are saving your model as an HDF5 file via `model.save()` or `keras.saving.save_model(model)`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')` or `keras.saving.save_model(model, 'my_model.keras')`.

External Test Data¶

In [48]:
# Loading required libraries
import librosa
import librosa.display
import os
In [41]:
import librosa
import numpy as np
import tensorflow as tf
import os
from collections import Counter


SAMPLE_RATE    = 32000
WINDOW_SIZE    = 10    
STEP_SIZE      = 1.5 
N_MELS         = 128
HOP_LENGTH     = 128
WIN_LENGTH     = 512
EXPECTED_WIDTH = 517

# ─────── Labels & Models ─────────────────────────────────────────────────────
species_labels = species  # your list of 12 species names

model1 = tf.keras.models.load_model("C:\\Users\\Nidhi\\Documents\\College Material\\Spring Quarter 2025\\SML2\Homework3\\bestmodel1.h5")
model2 = tf.keras.models.load_model("C:\\Users\\Nidhi\\Documents\\College Material\\Spring Quarter 2025\\SML2\Homework3\\bestmodel2.h5")
model3 = tf.keras.models.load_model("C:\\Users\\Nidhi\\Documents\\College Material\\Spring Quarter 2025\\SML2\Homework3\\bestmodel3.h5")
models = {"Model 1": model1, "Model 2": model2,"Model 3": model3}

mp3_files = [
    "C:\\Users\\Nidhi\\Documents\\College Material\\Spring Quarter 2025\\SML2\Homework3\\Homework 3_test_birds_test1.mp3",
    "C:\\Users\\Nidhi\\Documents\\College Material\\Spring Quarter 2025\\SML2\Homework3\\Homework 3_test_birds_test2.mp3",
    "C:\\Users\\Nidhi\\Documents\\College Material\\Spring Quarter 2025\\SML2\Homework3\\Homework 3_test_birds_test3.mp3",
]

def sliding_window_preds(audio, sr, model):
    window_len = int(sr * WINDOW_SIZE)
    step_len   = int(sr * STEP_SIZE)
    num_windows = max(1, int((len(audio) - window_len) / step_len) + 1)
    all_preds = []
    
    for i in range(num_windows):
        start, end = i*step_len, i*step_len + window_len
        y_win = audio[start:end].copy()
        
        # spectrogram
        mel = librosa.feature.melspectrogram(
            y=y_win, sr=sr,
            n_mels=N_MELS,
            hop_length=HOP_LENGTH,
            win_length=WIN_LENGTH
        )
        mel_db = librosa.power_to_db(mel, ref=np.max)
        norm  = (mel_db - mel_db.min()) / (mel_db.max() - mel_db.min())
        
        # resize to EXPECTED_WIDTH
        if norm.shape[1] > EXPECTED_WIDTH:
            norm = norm[:, :EXPECTED_WIDTH]
        else:
            pad_width = EXPECTED_WIDTH - norm.shape[1]
            norm = np.pad(norm, ((0,0),(0,pad_width)), mode='constant')
        
        # predict
        inp   = np.expand_dims(norm, axis=(0,-1))  # shape: (1,128,517,1)
        pred  = model.predict(inp, verbose=0)[0]   # vector of 12 probs
        all_preds.append(pred)
    
    return np.stack(all_preds)  # shape: (num_windows, 12)

WARNING:absl:Compiled the loaded model, but the compiled metrics have yet to be built. `model.compile_metrics` will be empty until you train or evaluate the model.
WARNING:absl:Compiled the loaded model, but the compiled metrics have yet to be built. `model.compile_metrics` will be empty until you train or evaluate the model.
WARNING:absl:Compiled the loaded model, but the compiled metrics have yet to be built. `model.compile_metrics` will be empty until you train or evaluate the model.
In [43]:
for model_idx, model in enumerate([model1, model2,model3], start=1):
    print(f"\n============================")
    print(f"Predictions using Model {model_idx}")
    print(f"============================")

    for file in mp3_files:
        print(f"\nFile: {os.path.basename(file)}")

        y, sr = librosa.load(file, sr=SAMPLE_RATE)
        preds = sliding_window_preds(y, sr, model)


        # Average and top predictions
        avg_pred = np.mean(preds, axis=0)
        top3_avg = np.argsort(avg_pred)[::-1][:3]

        print("Top 3 Average Predictions:")
        for i in top3_avg:
            print(f" - {species_labels[i]}: {avg_pred[i]:.2f}")

        # Count most frequent top-1 prediction per window
        top1_per_window = np.argmax(preds, axis=1)
        counts = Counter(top1_per_window)
        print("\nTop species per window:")
        for idx, cnt in counts.most_common():
            print(f" - {species_labels[idx]}: {cnt} windows")

        # Plot the segment with the strongest prediction
        best_idx = np.argmax(np.max(preds, axis=1))
        best_start = int(best_idx * SAMPLE_RATE * STEP_SIZE)
        best_end = best_start + int(SAMPLE_RATE * WINDOW_SIZE)
        y_best = y[best_start:best_end]

        mel = librosa.feature.melspectrogram(y=y_best, sr=sr, n_mels=N_MELS,
                                             hop_length=HOP_LENGTH, win_length=WIN_LENGTH)
        mel_db = librosa.power_to_db(mel, ref=np.max)

        plt.figure(figsize=(10, 4))
        librosa.display.specshow(mel_db, sr=sr, x_axis='time', y_axis='mel',
                                  cmap='gray_r', hop_length=HOP_LENGTH)
        plt.colorbar(format='%+2.0f dB')
        plt.title(f"Model {model_idx} - Strongest Segment - {os.path.basename(file)}")
        plt.tight_layout()
        plt.show()

============================
Predictions using Model 1
============================

File: Homework 3_test_birds_test1.mp3
Top 3 Average Predictions:
 - rewbla: 0.84
 - sonspa: 0.07
 - daejun: 0.06

Top species per window:
 - rewbla: 9 windows
No description has been provided for this image 
File: Homework 3_test_birds_test2.mp3
Top 3 Average Predictions:
 - rewbla: 0.59
 - daejun: 0.16
 - sonspa: 0.14

Top species per window:
 - rewbla: 1 windows
No description has been provided for this image 
File: Homework 3_test_birds_test3.mp3
Top 3 Average Predictions:
 - rewbla: 0.92
 - daejun: 0.04
 - sonspa: 0.03

Top species per window:
 - rewbla: 4 windows
No description has been provided for this image 
============================
Predictions using Model 2
============================

File: Homework 3_test_birds_test1.mp3
Top 3 Average Predictions:
 - houspa: 0.45
 - sonspa: 0.28
 - houfin: 0.27

Top species per window:
 - houspa: 9 windows
No description has been provided for this image 
File: Homework 3_test_birds_test2.mp3
Top 3 Average Predictions:
 - houspa: 0.42
 - sonspa: 0.29
 - houfin: 0.28

Top species per window:
 - houspa: 1 windows
No description has been provided for this image 
File: Homework 3_test_birds_test3.mp3
Top 3 Average Predictions:
 - houspa: 0.48
 - sonspa: 0.27
 - houfin: 0.25

Top species per window:
 - houspa: 4 windows
No description has been provided for this image 
============================
Predictions using Model 3
============================

File: Homework 3_test_birds_test1.mp3
Top 3 Average Predictions:
 - rewbla: 0.27
 - amerob: 0.21
 - amecro: 0.12

Top species per window:
 - rewbla: 9 windows
No description has been provided for this image 
File: Homework 3_test_birds_test2.mp3
Top 3 Average Predictions:
 - rewbla: 0.21
 - amerob: 0.21
 - amecro: 0.14

Top species per window:
 - rewbla: 1 windows
No description has been provided for this image 
File: Homework 3_test_birds_test3.mp3
Top 3 Average Predictions:
 - rewbla: 0.26
 - amerob: 0.23
 - amecro: 0.12

Top species per window:
 - amerob: 2 windows
 - rewbla: 2 windows
No description has been provided for this image
In [ ]: